package top.fzqblog.service;

import com.baomidou.mybatisplus.mapper.EntityWrapper;
import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import org.jsoup.Jsoup;
import org.jsoup.helper.StringUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.stereotype.Service;
import top.fzqblog.Utils.R;
import top.fzqblog.Utils.ThreadLocalClientFactory;
import top.fzqblog.model.CrawlerUser;
import top.fzqblog.model.ShareInfo;

import java.io.IOException;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * Created by 抽离 on 2017/7/18 0018.
 */
@Service
public class CrawlerShareService {

    private  String prefix = "http://pan.baidu.com";

    private AtomicInteger fileCount = new AtomicInteger(0);


    ExecutorService executorService = Executors.newFixedThreadPool(10);

    CompletionService<Integer> cService = new ExecutorCompletionService<Integer>(executorService);

    public void crawler(){
        List<CrawlerUser> crawlerUsers = new CrawlerUser().selectList(new EntityWrapper<CrawlerUser>().eq("uk", "1292902436"));
        Future<Map<String, Object>> future = null;

        for(CrawlerUser crawlerUser : crawlerUsers){
            String url = "http://pan.baidu.com/wap/share/home?third=0&uk=" + crawlerUser.getUk() + "&start=3200";
            System.out.println("url：" + url);
            future = executorService.submit(new CrawlerShare(url));
        }
//        executorService.shutdown();
        try
        {
            // awaitTermination返回false即超时会继续循环，返回true即线程池中的线程执行完成主线程跳出循环往下执行，每隔10秒循环一次
            while (!executorService.awaitTermination(10, TimeUnit.SECONDS))
            {

            }
        }
        catch (InterruptedException e)
        {
            e.printStackTrace();
        }
        System.out.println("总共有：" + fileCount + "个");

    }


    public void crawlerByCompletableService(){
        List<CrawlerUser> crawlerUsers = new CrawlerUser()
                .selectAll();
//                .selectList(new EntityWrapper<CrawlerUser>().eq("uk", "1292902436"));

        for(CrawlerUser crawlerUser : crawlerUsers){
            cService.submit(new ShareCallable("http://pan.baidu.com/wap/share/home?third=0&uk=" + crawlerUser.getUk()));
        }
        long begin = System.currentTimeMillis();
//        boolean flag = true;
        int  count = crawlerUsers.size();
        System.out.println("共有：" + count + "个用户等待抓取");
        while(count > 0){
            try {
                Future<Integer> future = cService.take();
//                System.out.println("：" + future.get());
                if(future.get() == 1){
                    count --;
                    System.out.println("还剩：" + count + "个用户没抓取");
                }
            } catch (InterruptedException | ExecutionException e) {
                e.printStackTrace();
            }
        }
        System.out.println("执行完毕，花费：" + (System.currentTimeMillis() - begin)/(1000 * 60) + "分钟");
        executorService.shutdown();
    }


    class ShareCallable implements Callable<Integer> {

        private String url;

        public ShareCallable(String url) {
            this.url = url;
        }

        @Override
        public Integer call() throws Exception {
            System.out.println("url：" + url);
            WebClient webClient = ThreadLocalClientFactory.getInstance().getWebClient();

             webClient.getOptions().setJavaScriptEnabled(true);
            try {
                HtmlPage htmlPage = webClient.getPage(url);
//                webClient.waitForBackgroundJavaScript(3000);
//                Thread.sleep(5000);
                for(int i = 0; i < 3; i++){
                    htmlPage = webClient.getPage(url);
                    Thread.sleep(1000);
                }
                Document document = Jsoup.parse(htmlPage.asXml());
//                System.out.println("doc：" + document);
                if("页面不存在".equals(document.title())){
                    cService.submit(new ShareCallable(url));
                    return 0;
                }
                Elements elements = document.select(".list-item");
                for (Element element : elements){
                    ShareInfo shareInfo = new ShareInfo();
                    shareInfo.setCreateTime(new Date());
                    shareInfo.setFileUrl(prefix + element.attr("href"));
                    shareInfo.setFileName(element.select(".content h3").text());
                    shareInfo.setFileType(element.select(".fn-suffix").text());
                    Optional.ofNullable(element.select(".list-content").text())
                            .ifPresent(
                                    date ->{
                                        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                        try {
                                            shareInfo.setShareTime(format.parse(date.trim()));
                                        } catch (ParseException e) {
                                            e.printStackTrace();
                                        }
                                    }
                            );
                    shareInfo.insert();
                    fileCount.incrementAndGet();
                }
                webClient.close();
                Elements next = document.select("a[node-type=next-page]");
                String nextUrl = null;
//                System.out.println("next：" + next.toString());
                if(!StringUtil.isBlank(next.toString())){
                    Element nextElement = next.get(0);
                    if(!StringUtil.isBlank(nextElement.toString())){
                        nextUrl = nextElement.attr("href");
                    }
                }
                if(nextUrl == null){
                    return 1;
                }
                else{
                    System.out.println("nextUrl：" + nextUrl);
                    cService.submit(new ShareCallable(prefix + nextUrl));
                    return 0;
                }
            } catch (IOException e) {
                e.printStackTrace();
                cService.submit(new ShareCallable(url));
                return 0;
            }
        }
    }

    class CrawlerShare implements Callable<Map<String, Object>> {

        private String url;


        public CrawlerShare(String url) {
            this.url = url;
        }

        @Override
        public Map<String, Object> call() throws Exception {
            System.out.println("url：" + url);
            WebClient webClient = new WebClient(BrowserVersion.FIREFOX_52);
//             webClient.getOptions().setJavaScriptEnabled(false);
            try {
                HtmlPage htmlPage = webClient.getPage(url);
                webClient.waitForBackgroundJavaScript(1000);
//                Thread.sleep(5000);
                Document document = Jsoup.parse(htmlPage.asXml());
                Elements elements = document.select(".list-item");
                for (Element element : elements){
                    ShareInfo shareInfo = new ShareInfo();
                    shareInfo.setCreateTime(new Date());
                    shareInfo.setFileUrl(prefix + element.attr("href"));
                    shareInfo.setFileName(element.select(".content h3").text());
                    shareInfo.setFileType(element.select(".fn-suffix").text());
                    Optional.ofNullable(element.select(".list-content").text())
                            .ifPresent(
                                    date ->{
                                        SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
                                        try {
                                            shareInfo.setShareTime(format.parse(date.trim()));
                                        } catch (ParseException e) {
                                            e.printStackTrace();
                                        }
                                    }
                            );
                    shareInfo.insert();
                    fileCount.incrementAndGet();
                }
                webClient.close();
                Optional.ofNullable(document.select("a[node-type=next-page]").attr("href"))
                        .ifPresent(
                                nextUrl ->{
                                    if(!StringUtil.isBlank(nextUrl)) {
                                        System.out.println("nextUrl====================：" + nextUrl);
                                        nextUrl = prefix + nextUrl;
                                        executorService.submit(new CrawlerShare(nextUrl));
                                    }
                                }
                        );
                return R.ok();
            } catch (IOException e) {
                R.error();
                e.printStackTrace();
            }
            return R.error();
        }
    }

}
